{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "51cb8263-3a44-404d-aa99-ace5510525eb",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/sparksampling/venv/lib/python3.8/site-packages/pandas/compat/__init__.py:120: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "from sparksampling.config import SPARK_CONF\n",
    "\n",
    "conf = SPARK_CONF\n",
    "spark = SparkSession.builder.config(conf=conf).getOrCreate()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d243ef93-5dae-4bba-a848-4650d5d18af1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = spark.read.csv(\"ten_million_top1k.csv\", header=True).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2393c2f8-3b4e-44af-b8c3-36b900149621",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     y\n",
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "3    1\n",
       "4    0\n",
       "..  ..\n",
       "995  0\n",
       "996  1\n",
       "997  0\n",
       "998  0\n",
       "999  1\n",
       "\n",
       "[1000 rows x 1 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = df[['y']]\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "35e108ef-ce8b-414c-b84e-68c9c9dca575",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['6', '5', '4', ..., '-3.6292718', '2.1533357000000004',\n",
       "        '0.21443366'],\n",
       "       ['6', '2', '0', ..., '0.61825443', '-0.20594382',\n",
       "        '1.0061347999999999'],\n",
       "       ['4', '2', '0', ..., '-0.35092328', '0.35849345', '0.91475412'],\n",
       "       ...,\n",
       "       ['6', '2', '3', ..., '0.17274349', '0.083568695', '0.85332849'],\n",
       "       ['6', '5', '3', ..., '1.7405881', '-0.81428724', '-0.93481414'],\n",
       "       ['0', '6', '2', ..., '0.42450829999999995',\n",
       "        '-0.0017179298999999999', '-3.8725715']], dtype=object)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df.drop(columns=['# id','y'], axis=1)\n",
    "X.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b016ba39-d06d-48bd-927e-b18b4b30e4e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import EditedNearestNeighbours\n",
    "enn = EditedNearestNeighbours()\n",
    "x_fit, y_fit = enn.fit_resample(X.values, y.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "35a9d9f2-8221-4a95-b5ef-760ee5be7e03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X_0</th>\n",
       "      <th>X_1</th>\n",
       "      <th>X_2</th>\n",
       "      <th>X_3</th>\n",
       "      <th>X_4</th>\n",
       "      <th>X_5</th>\n",
       "      <th>X_6</th>\n",
       "      <th>X_7</th>\n",
       "      <th>X_8</th>\n",
       "      <th>X_9</th>\n",
       "      <th>...</th>\n",
       "      <th>X_90</th>\n",
       "      <th>X_91</th>\n",
       "      <th>X_92</th>\n",
       "      <th>X_93</th>\n",
       "      <th>X_94</th>\n",
       "      <th>X_95</th>\n",
       "      <th>X_96</th>\n",
       "      <th>X_97</th>\n",
       "      <th>X_98</th>\n",
       "      <th>X_99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.713142</td>\n",
       "      <td>2.951669</td>\n",
       "      <td>-1.608283</td>\n",
       "      <td>-2.687212</td>\n",
       "      <td>-0.809006</td>\n",
       "      <td>0.679043</td>\n",
       "      <td>2.420450</td>\n",
       "      <td>-3.629272</td>\n",
       "      <td>2.153336</td>\n",
       "      <td>0.214434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.223221</td>\n",
       "      <td>0.446508</td>\n",
       "      <td>0.651251</td>\n",
       "      <td>-0.727004</td>\n",
       "      <td>2.328492</td>\n",
       "      <td>0.697474</td>\n",
       "      <td>0.656398</td>\n",
       "      <td>0.618254</td>\n",
       "      <td>-0.205944</td>\n",
       "      <td>1.006135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.015165</td>\n",
       "      <td>1.172190</td>\n",
       "      <td>0.741630</td>\n",
       "      <td>-1.366551</td>\n",
       "      <td>-0.656233</td>\n",
       "      <td>1.058246</td>\n",
       "      <td>1.232354</td>\n",
       "      <td>-0.350923</td>\n",
       "      <td>0.358493</td>\n",
       "      <td>0.914754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.333362</td>\n",
       "      <td>1.060832</td>\n",
       "      <td>-1.199539</td>\n",
       "      <td>-0.961484</td>\n",
       "      <td>2.144502</td>\n",
       "      <td>0.723086</td>\n",
       "      <td>0.866016</td>\n",
       "      <td>-1.320033</td>\n",
       "      <td>0.781048</td>\n",
       "      <td>0.159866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.972420</td>\n",
       "      <td>0.897007</td>\n",
       "      <td>-0.759644</td>\n",
       "      <td>-1.092099</td>\n",
       "      <td>-0.534969</td>\n",
       "      <td>-0.384158</td>\n",
       "      <td>0.985030</td>\n",
       "      <td>-0.099703</td>\n",
       "      <td>0.197429</td>\n",
       "      <td>-0.078188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>943</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.696159</td>\n",
       "      <td>-0.839109</td>\n",
       "      <td>-1.309599</td>\n",
       "      <td>1.005017</td>\n",
       "      <td>1.649397</td>\n",
       "      <td>-1.136458</td>\n",
       "      <td>-0.906425</td>\n",
       "      <td>0.153697</td>\n",
       "      <td>-0.212211</td>\n",
       "      <td>-2.378260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>944</th>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.218904</td>\n",
       "      <td>-1.410298</td>\n",
       "      <td>-0.492720</td>\n",
       "      <td>1.699682</td>\n",
       "      <td>-0.261111</td>\n",
       "      <td>-0.654157</td>\n",
       "      <td>-1.532982</td>\n",
       "      <td>0.219928</td>\n",
       "      <td>-0.339177</td>\n",
       "      <td>0.989774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>945</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.204937</td>\n",
       "      <td>-1.631751</td>\n",
       "      <td>1.633170</td>\n",
       "      <td>2.091174</td>\n",
       "      <td>0.541196</td>\n",
       "      <td>-0.266991</td>\n",
       "      <td>-1.886537</td>\n",
       "      <td>-0.199328</td>\n",
       "      <td>-0.185737</td>\n",
       "      <td>0.185880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.881256</td>\n",
       "      <td>0.427251</td>\n",
       "      <td>1.080183</td>\n",
       "      <td>-0.409996</td>\n",
       "      <td>-0.011785</td>\n",
       "      <td>0.343887</td>\n",
       "      <td>0.369398</td>\n",
       "      <td>-0.448761</td>\n",
       "      <td>0.276815</td>\n",
       "      <td>0.651438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947</th>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.642889</td>\n",
       "      <td>1.130855</td>\n",
       "      <td>0.312951</td>\n",
       "      <td>-1.527879</td>\n",
       "      <td>0.745244</td>\n",
       "      <td>-0.502668</td>\n",
       "      <td>1.378637</td>\n",
       "      <td>0.424508</td>\n",
       "      <td>-0.001718</td>\n",
       "      <td>-3.872571</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>948 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     X_0  X_1  X_2  X_3  X_4  X_5  X_6  X_7  X_8  X_9  ...      X_90  \\\n",
       "0    6.0  5.0  4.0  5.0  2.0  2.0  3.0  3.0  6.0  5.0  ... -1.713142   \n",
       "1    6.0  2.0  0.0  4.0  5.0  2.0  6.0  3.0  6.0  0.0  ...  1.223221   \n",
       "2    4.0  2.0  0.0  4.0  6.0  3.0  3.0  3.0  6.0  0.0  ...  1.015165   \n",
       "3    4.0  6.0  3.0  5.0  2.0  2.0  6.0  1.0  1.0  0.0  ... -2.333362   \n",
       "4    0.0  5.0  2.0  4.0  6.0  2.0  3.0  5.0  2.0  1.0  ... -2.972420   \n",
       "..   ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...       ...   \n",
       "943  3.0  0.0  5.0  4.0  6.0  5.0  4.0  2.0  1.0  1.0  ... -0.696159   \n",
       "944  0.0  6.0  5.0  4.0  6.0  5.0  6.0  6.0  0.0  4.0  ... -0.218904   \n",
       "945  1.0  0.0  0.0  4.0  6.0  6.0  6.0  3.0  1.0  1.0  ...  0.204937   \n",
       "946  1.0  4.0  5.0  3.0  5.0  6.0  2.0  1.0  3.0  2.0  ... -0.881256   \n",
       "947  0.0  6.0  2.0  0.0  1.0  2.0  2.0  2.0  1.0  2.0  ... -0.642889   \n",
       "\n",
       "         X_91      X_92      X_93      X_94      X_95      X_96      X_97  \\\n",
       "0    2.951669 -1.608283 -2.687212 -0.809006  0.679043  2.420450 -3.629272   \n",
       "1    0.446508  0.651251 -0.727004  2.328492  0.697474  0.656398  0.618254   \n",
       "2    1.172190  0.741630 -1.366551 -0.656233  1.058246  1.232354 -0.350923   \n",
       "3    1.060832 -1.199539 -0.961484  2.144502  0.723086  0.866016 -1.320033   \n",
       "4    0.897007 -0.759644 -1.092099 -0.534969 -0.384158  0.985030 -0.099703   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "943 -0.839109 -1.309599  1.005017  1.649397 -1.136458 -0.906425  0.153697   \n",
       "944 -1.410298 -0.492720  1.699682 -0.261111 -0.654157 -1.532982  0.219928   \n",
       "945 -1.631751  1.633170  2.091174  0.541196 -0.266991 -1.886537 -0.199328   \n",
       "946  0.427251  1.080183 -0.409996 -0.011785  0.343887  0.369398 -0.448761   \n",
       "947  1.130855  0.312951 -1.527879  0.745244 -0.502668  1.378637  0.424508   \n",
       "\n",
       "         X_98      X_99  \n",
       "0    2.153336  0.214434  \n",
       "1   -0.205944  1.006135  \n",
       "2    0.358493  0.914754  \n",
       "3    0.781048  0.159866  \n",
       "4    0.197429 -0.078188  \n",
       "..        ...       ...  \n",
       "943 -0.212211 -2.378260  \n",
       "944 -0.339177  0.989774  \n",
       "945 -0.185737  0.185880  \n",
       "946  0.276815  0.651438  \n",
       "947 -0.001718 -3.872571  \n",
       "\n",
       "[948 rows x 100 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.DataFrame(x_fit, columns = X.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c23e8add-6cd9-4def-8a91-dfec476c434d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>943</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>944</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>945</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>948 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     y\n",
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "3    0\n",
       "4    0\n",
       "..  ..\n",
       "943  1\n",
       "944  1\n",
       "945  1\n",
       "946  1\n",
       "947  1\n",
       "\n",
       "[948 rows x 1 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(y_fit, columns = y.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "dfd8a825-bc96-4a33-9415-39946581e777",
   "metadata": {},
   "outputs": [],
   "source": [
    "resultdf = pd.concat([pd.DataFrame(x_fit, columns = X.columns), pd.DataFrame(y_fit, columns = y.columns)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "055eaab3-c2eb-4de3-ac04-56659e005d96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[X_0: double, X_1: double, X_2: double, X_3: double, X_4: double, X_5: double, X_6: double, X_7: double, X_8: double, X_9: double, X_10: double, X_11: double, X_12: double, X_13: double, X_14: double, X_15: double, X_16: double, X_17: double, X_18: double, X_19: double, X_20: double, X_21: double, X_22: double, X_23: double, X_24: double, X_25: double, X_26: double, X_27: double, X_28: double, X_29: double, X_30: double, X_31: double, X_32: double, X_33: double, X_34: double, X_35: double, X_36: double, X_37: double, X_38: double, X_39: double, X_40: double, X_41: double, X_42: double, X_43: double, X_44: double, X_45: double, X_46: double, X_47: double, X_48: double, X_49: double, X_50: double, X_51: double, X_52: double, X_53: double, X_54: double, X_55: double, X_56: double, X_57: double, X_58: double, X_59: double, X_60: double, X_61: double, X_62: double, X_63: double, X_64: double, X_65: double, X_66: double, X_67: double, X_68: double, X_69: double, X_70: double, X_71: double, X_72: double, X_73: double, X_74: double, X_75: double, X_76: double, X_77: double, X_78: double, X_79: double, X_80: double, X_81: double, X_82: double, X_83: double, X_84: double, X_85: double, X_86: double, X_87: double, X_88: double, X_89: double, X_90: double, X_91: double, X_92: double, X_93: double, X_94: double, X_95: double, X_96: double, X_97: double, X_98: double, X_99: double, y: string]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.createDataFrame(resultdf)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
